/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2019-2022. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * Description:
 * Author: huawei
 * Create: 2019-10-15
 */

#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/version.h>
#include <linux/list.h>
#include <linux/atomic.h>
#include <linux/nsproxy.h>

#include "svm_cmd.h"
#include "devmm_proc_info.h"
#include "devmm_proc_mem_copy.h"
#include "devmm_channel.h"
#include "devdrv_interface.h"
#include "devmm_common.h"
#include "devmm_dev.h"
#include "svm_proc_mng.h"
#include "svm_mem_mng.h"
#include "svm_heap_mng.h"
#include "svm_register_ops.h"
#include "svm_srcu_work.h"

#ifdef CFG_FEATURE_VFIO
#include "devmm_pm_vpc.h"
#include "devmm_pm_adapt.h"
#endif

STATIC int devmm_svm_open(struct inode *inode, struct file *file)
{
    file->private_data = kzalloc(sizeof(struct devmm_private_data), GFP_KERNEL | __GFP_ACCOUNT);
    if (file->private_data == NULL) {
        devmm_drv_err("Kzalloc devmm_private_data fail.\n");
        return -ENOMEM;
    }
    return 0;
}

STATIC int _devmm_svm_mmap_config_svm_proc(struct devmm_svm_process *svm_proc, struct vm_area_struct *vma)
{
    if ((svm_proc->vma_num >= devmm_svm->mmap_para.mmap_num) ||
        (svm_proc->inited != DEVMM_SVM_PRE_INITING_FLAG)) {
        devmm_drv_err("Svm map get_svm_process error. "
            "(vm_start=0x%lx; vm_end=0x%lx; vm_pgoff=0x%lx; vm_flags=0x%lx)\n",
            vma->vm_start, vma->vm_end, vma->vm_pgoff, vma->vm_flags);
        return -ESRCH;
    }
    devmm_remove_vma_wirte_flag(vma);
    /* init vma */
    svm_proc->vma[svm_proc->vma_num] = vma;
    svm_proc->vma_num++;
    if (svm_proc->vma_num == devmm_svm->mmap_para.mmap_num) {
        svm_proc->mm = current->mm;
        svm_proc->tsk = current;
        devmm_svm_mem_enable(svm_proc);
        devmm_set_svm_proc_state(svm_proc, DEVMM_SVM_INITING_FLAG);
        devmm_drv_info("Devmm_map success. (hostpid=%d; devid=%d; vfid=%d; devpid=%d; status=%u; proc_idx=%u)\n",
            svm_proc->process_id.hostpid, svm_proc->process_id.devid, svm_proc->process_id.vfid,
            svm_proc->devpid, svm_proc->notifier_reg_flag, svm_proc->proc_idx);
    }

    return 0;
}

STATIC int devmm_svm_mmap_config_svm_proc(struct devmm_svm_process *svm_proc, struct vm_area_struct *vma)
{
    int ret;

    mutex_lock(&svm_proc->proc_lock);
    ret = _devmm_svm_mmap_config_svm_proc(svm_proc, vma);
    mutex_unlock(&svm_proc->proc_lock);

    return ret;
}

STATIC int devmm_svm_mmap_check_para(struct file *file, struct vm_area_struct *vma)
{
    struct devmm_private_data *private_data = (struct devmm_private_data *)file->private_data;

    if (private_data == NULL) {
        devmm_drv_err("Unexpected, private_data is NULL.\n");
        return -EINVAL;
    }

    if ((vma->vm_start < DEVMM_SVM_MEM_START) || (vma->vm_end > (DEVMM_SVM_MEM_START + DEVMM_SVM_MEM_SIZE))) {
        devmm_drv_err("Svm map va not fixed. (vm_start=0x%lx; vm_end=0x%lx; vm_pgoff=0x%lx; vm_flags=0x%lx)\n",
            vma->vm_start, vma->vm_end, vma->vm_pgoff, vma->vm_flags);
        return -EINVAL;
    }

    if ((private_data->custom_flag == 0) && (private_data->process == NULL)) {
        devmm_drv_err("File's svm_proc is null, please call alloc_svm_proc first.\n");
        return -EINVAL;
    }

    return 0;
}

STATIC int devmm_svm_mmap(struct file *file, struct vm_area_struct *vma)
{
    struct devmm_svm_process *svm_proc = NULL;
    int ret;

    ret = devmm_svm_mmap_check_para(file, vma);
    if (ret != 0) {
        devmm_drv_err("Svm map para check error. (ret=%d)\n", ret);
        return ret;
    }

    vm_flags_set(vma,(VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_PFNMAP | VM_LOCKED | VM_WRITE));
    //vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_PFNMAP | VM_LOCKED | VM_WRITE;
    devmm_svm_setup_vma_ops(vma);

    if (((struct devmm_private_data *)file->private_data)->custom_flag == 0) {
        svm_proc = ((struct devmm_private_data *)file->private_data)->process;
        ret = devmm_svm_mmap_config_svm_proc(svm_proc, vma);
        if (ret != 0) {
            devmm_drv_err("Svm map config_svm_proc error. (vm_start=0x%lx; vm_end=0x%lx)\n",
                vma->vm_start, vma->vm_end);
            return ret;
        }
    }

    return 0;
}

STATIC int devmm_svm_release(struct inode *inode, struct file *filp)
{
    struct devmm_svm_process *svm_proc = NULL;

    if (filp->private_data == NULL) {
        devmm_drv_info("Private_data is NULL.\n");
        return 0;
    }
    if (((struct devmm_private_data *)filp->private_data)->custom_flag != 0) {
        devmm_drv_info("Custom exit.\n");
        goto free;
    }
    if (((struct devmm_private_data *)filp->private_data)->process == NULL) {
        devmm_drv_info("Process is NULL.\n");
        goto free;
    }
    svm_proc = (struct devmm_svm_process *)(((struct devmm_private_data *)filp->private_data)->process);
    devmm_svm_mem_disable(svm_proc);
    devmm_svm_release_proc(svm_proc);
free:
    kfree(filp->private_data);
    filp->private_data = NULL;

    return 0;
}

STATIC int devmm_ioctl_get_svm_proc_from_file(struct file *file, u32 cmd, struct devmm_svm_process **svm_proc)
{
    if (_IOC_NR(cmd) < DEVMM_SVM_CMD_USE_PRIVATE_MAX_CMD) {
        *svm_proc = devmm_get_svm_proc_from_file(file);
        if (*svm_proc == NULL ||
            (devmm_get_end_type() == DEVMM_END_HOST && cmd != DEVMM_SVM_INIT_PROCESS &&
            ((*svm_proc)->inited != DEVMM_SVM_INITED_FLAG ||
            (*svm_proc)->process_id.hostpid != devmm_get_current_pid()))) {
            devmm_drv_err("Invalid svm_proc states.");
            return -EINVAL;
        }
    }
    return 0;
}

static int devmm_dispatch_ioctl_use_svm_proc(struct devmm_svm_process *svm_proc,
    u32 cmd, struct devmm_ioctl_arg *buffer)
{
    u32 cmd_id = _IOC_NR(cmd);
    u32 cmd_flag;
    int ret = 0;

    if (devmm_ioctl_handlers[cmd_id].ioctl_handler == NULL) {
        devmm_drv_err("Cmd not support. (cmd=0x%x; cmd_id=0x%x)\n", cmd, cmd_id);
        return -EOPNOTSUPP;
    }
    cmd_flag = devmm_ioctl_handlers[cmd_id].cmd_flag;
    ret = devmm_convert_id_from_vir_to_phy(svm_proc, buffer, cmd_flag);
    if (ret != 0) {
        devmm_drv_err("Virtual id to physical id failed. (cmd=0x%x; cmd_id= 0x%x; ret=%d)\n", cmd, cmd_id, ret);
        return ret;
    }
    ret = devmm_check_cmd_support(cmd_flag);
    if (ret != 0) {
        devmm_drv_err("Not support cmd. (cmd=0x%x; cmd_id=0x%x)\n", cmd_flag, cmd_id);
        return ret;
    }

    devmm_svm_ioctl_lock(svm_proc, cmd_flag);
    ret = devmm_ioctl_dispatch(svm_proc, cmd_id, cmd_flag, buffer);
    devmm_svm_ioctl_unlock(svm_proc, cmd_flag);
    return ret;
}

STATIC int devmm_dispatch_ioctl_normal(struct file *file, u32 cmd, struct devmm_ioctl_arg *buffer)
{
    struct devmm_svm_process *svm_proc = NULL;
    int ret;

    ret = devmm_ioctl_get_svm_proc_from_file(file, cmd, &svm_proc);
    if (ret != 0) {
        devmm_drv_err("Get svm_proc failed. (cmd=0x%x; _IOC_NR(cmd)=0x%x)\n", cmd, _IOC_NR(cmd));
        return ret;
    }
    ret = devmm_dispatch_ioctl_use_svm_proc(svm_proc, cmd, buffer);
    if (ret != 0) {
        return ret;
    }
    return 0;
}

STATIC int devmm_dispatch_ioctl_for_file_arg(struct file *file, u32 cmd, struct devmm_ioctl_arg *buffer)
{
    return devmm_ioctl_file_arg_handlers[_IOC_NR(cmd)](file, buffer);
}

STATIC bool devmm_ioctl_cmd_is_file_arg(u32 cmd)
{
    return (devmm_ioctl_file_arg_handlers[_IOC_NR(cmd)] != NULL) ? true : false;
}

STATIC int devmm_dispatch_ioctl(struct file *file, u32 cmd, struct devmm_ioctl_arg *buffer)
{
    if (devmm_ioctl_cmd_is_file_arg(cmd) == true) {
        return devmm_dispatch_ioctl_for_file_arg(file, cmd, buffer);
    } else {
        return devmm_dispatch_ioctl_normal(file, cmd, buffer);
    }
}

STATIC long devmm_svm_ioctl(struct file *file, u32 cmd, unsigned long arg)
{
    struct devmm_ioctl_arg buffer = {{0}};
    u32 cmd_id = _IOC_NR(cmd);
    int ret;

    if ((file == NULL) || (file->private_data == NULL) || (arg == 0)) {
        devmm_drv_err("File is NULL, check svm init. (cmd=0x%x; _IOC_NR(cmd)=0x%x)\n", cmd, _IOC_NR(cmd));
        return -EINVAL;
    }

    if ((_IOC_TYPE(cmd) != DEVMM_SVM_MAGIC) || (cmd_id >= DEVMM_SVM_CMD_MAX_CMD)) {
        devmm_drv_err("Cmd not support. (cmd=0x%x; cmd_id=0x%x)\n", cmd, cmd_id);
        return -EINVAL;
    }

    if ((_IOC_DIR(cmd) & _IOC_WRITE) != 0) {
        if (copy_from_user(&buffer, (void __user *)(uintptr_t)arg, sizeof(struct devmm_ioctl_arg)) != 0) {
            devmm_drv_err("Copy_from_user fail. (cmd=0x%x; cmd_id=0x%x)\n", cmd, cmd_id);
            return -EINVAL;
        }
    }

    ret = devmm_dispatch_ioctl(file, cmd, &buffer);
    if (ret != 0) {
        return ret;
    }

    if ((_IOC_DIR(cmd) & _IOC_READ) != 0) {
        if (copy_to_user((void __user *)(uintptr_t)arg, &buffer, sizeof(struct devmm_ioctl_arg)) != 0) {
            devmm_drv_err("Copy_to_user fail. (cmd=0x%x; cmd_id=0x%x)\n", cmd, cmd_id);
            return -EINVAL;
        }
    }

    return 0;
}

STATIC struct file_operations devmm_svm_fops = {
    .owner = THIS_MODULE,
    .open = devmm_svm_open,
    .release = devmm_svm_release,
    .mmap = devmm_svm_mmap,
    .unlocked_ioctl = devmm_svm_ioctl,
};

/*
 * devmm svm module init
 */
struct devmm_svm_dev *devmm_svm = NULL;

STATIC int devmm_alloc_svm_struct(void)
{
    u32 i;

    devmm_svm = (struct devmm_svm_dev *)vzalloc(sizeof(struct devmm_svm_dev));
    if (devmm_svm == NULL) {
        devmm_drv_err("Vzalloc svm fail.\n");
        return -ENOMEM;
    }

    devmm_svm->dev_no = 0;
    devmm_svm->host_page_shift = 0;
    devmm_svm->device_page_shift = 0;
    devmm_svm->svm_page_size = PAGE_SIZE;
#ifdef HOST_AGENT
    devmm_svm->device_page_size = PAGE_SIZE;
#endif
    devmm_svm->page_size_inited = 0;
    devmm_svm->smmu_status = DEVMM_SMMU_STATUS_UNINIT;
#ifdef CFG_FEATURE_VFIO
    for (i = 0; i < DEVMM_MAX_DEVICE_NUM; i++) {
        devmm_svm->total_convert_len[i] = DEVMM_VDEV_MAX_CONVERT_LEN_DEFAULT;
    }
#endif

    for (i = 0; i < DEVMM_MAX_DEVICE_NUM; i++) {
        devmm_svm->device_info.cluster_id[i] = DEVMM_MAX_DEVICE_NUM;
    }
    init_rwsem(&devmm_svm->convert_sem);
    atomic64_set(&devmm_svm->device_info.total_ddr, 0);
    atomic64_set(&devmm_svm->device_info.total_hbm, 0);
    for (i = 0; i < HOST_REGISTER_MAX_TPYE; i++) {
        INIT_LIST_HEAD(&devmm_svm->shm_pro_head[i].head);
        mutex_init(&devmm_svm->shm_pro_head[i].node_lock);
    }
    mutex_init(&devmm_svm->setup_lock);

    return 0;
}

STATIC void devmm_free_svm_struct(void)
{
    vfree(devmm_svm);
    devmm_svm = NULL;
}

STATIC int devmm_init_devmm_struct(void)
{
    if (devmm_alloc_svm_struct() != 0) {
        devmm_drv_err("Svm struct alloc fail.\n");
        return -ENOMEM;
    }

    devmm_init_dev_private(devmm_svm, &devmm_svm_fops);

    devmm_default_srcu_work_init();

    return devmm_svm_proc_mng_init();
}

STATIC void devmm_unint_devmm_struct(void)
{
    devmm_svm_proc_mng_uinit();
    devmm_free_svm_struct();
}

#ifndef DEVMM_UT
#ifdef HOST_AGENT
STATIC void devmm_svm_set_host_agent_pgsf(void)
{
    devmm_svm_set_host_pgsf(PAGE_SHIFT);
    devmm_svm_set_host_hpgsf(HPAGE_SHIFT);

    devmm_svm_set_device_pgsf(PAGE_SHIFT);
    devmm_svm_set_device_hpgsf(HPAGE_SHIFT);
    devmm_chan_set_host_device_page_size();
}
#endif
#endif

STATIC int devmm_svm_init(void)
{
    int ret;

    devmm_drv_info("Devmm model init. (svm_dev_size=%lu)\n", sizeof(struct devmm_svm_dev));

    ret = devmm_init_devmm_struct();
    if (ret != 0) {
        devmm_drv_err("Devmm_init_devmm_struct fail.\n");
        return -ENOMEM;
    }

    ret = devmm_svm_davinci_module_init(&devmm_svm_fops);
    if (ret != 0) {
        devmm_drv_err("Devmm_init_devmm_struct fail.\n");
        goto register_davinci_fail;
    }

    ret = devmm_svm_dev_init(&devmm_svm_fops);
    if (ret != 0) {
        devmm_drv_err("Alloc_chrdev_region error.\n");
        goto register_dev_fail;
    }

#ifdef CFG_FEATURE_VFIO
    ret = vdevmmh_init();
    if (ret != 0) {
        devmm_drv_err("Register vdevmmh client error. (ret=%d)\n", ret);
        goto register_client_fail;
    }
#endif

    ret = devmm_register_ops_init();
    if (ret != 0) {
        devmm_drv_err("Register client error. (ret=%d)\n", ret);
#ifdef CFG_FEATURE_VFIO
        vdevmmh_uninit();
#endif
        goto register_client_fail;
    }

#ifdef HOST_AGENT
#ifndef DEVMM_UT
    devmm_svm_set_host_agent_pgsf();
#endif
#endif

    devmm_drv_info("Devmm model init success.\n");
    return 0;

register_client_fail:
    devmm_svm_dev_destory();
register_dev_fail:
    devmm_svm_davinci_module_uninit();
register_davinci_fail:
    devmm_unint_devmm_struct();

    return ret;
}

STATIC void devmm_svm_exit(void)
{
#ifdef CFG_FEATURE_VFIO
    /*
     * vm msg_chan init need use vpc chan to sync pm info
     * rmmod ko pm proc release need use msg chan
     * so init and destroy vdevmmh first
     */
    vdevmmh_uninit();
#endif

    devmm_unregister_ops_uninit();

    devmm_uninit_dev_private(devmm_svm);

    devmm_svm_dev_destory();
    devmm_svm_davinci_module_uninit();
    devmm_unint_devmm_struct();
}

module_init(devmm_svm_init);
module_exit(devmm_svm_exit);
MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("devmm shared memory manager driver");
